import random
import time
import csv
import requests
from bs4 import BeautifulSoup
import json

from config import proxies, url, headers

bs = BeautifulSoup


def write_to_f(f_name, content):
    with open(f'file_{f_name}.json', 'a', encoding='utf-8') as f:
        json.dump(content, f, indent=4)


def get_response(url):
    response = requests.get(url=url,
                            headers=headers,
                            proxies=proxies)
    write_to_f('main_page.html', response.text)

    dict_hrefs = []

    with open('file_main_page.html', 'r', encoding='utf-8') as f:
        file = f.read()

    soup = bs(file, 'lxml')
    find_columns = soup.find_all('div', class_='catalog-column')
    for el in find_columns:
        ul = el.find('ul')
        li = ul.find_all('li')
        print(li)
        for content in li:
            text = content.find('a').text
            href = content.find('a').get('href')
            dict_hrefs.append({'name': text,
                               'href': href})

    write_to_f('hrefs', dict_hrefs)


def get_hrefs_cars(url):
    with open('file_hrefs.json', 'r', encoding='utf-8') as f:
        hrefs = json.load(f)

    for el in hrefs:
        name = el['name']
        text_href = []
        text_href.clear()
        time.sleep(1)
        response = requests.get(url=f'{url}{el["href"]}',
                                headers=random.choice(headers),
                                proxies=random.choice(proxies))
        soup = bs(response.text, 'lxml')
        models = soup.find('div', id='models').find_all('div', class_='cell2')
        for model in models:
            model = model.find('div').find('div', class_='car-info__description').find(
                'div'
            ).find('a')
            text = model.text
            href = model.get('href')
            t_h = text, href
            text_href.append(t_h)

        with open(f'models_brand_car/{name}_models.csv', 'w', encoding='utf-8') as f:
            writer = csv.writer(f)
            writer.writerow(
              ['model', 'link'],
                )

        for content_model in text_href:
            with open(f'models_brand_car/{name}_models.csv', 'a', encoding='utf-8') as f:
                writer = csv.writer(f)
                writer.writerow(
                    content_model
                )


def main():
    get_response(url=url)
    get_hrefs_cars(url=url)


if __name__ == '__main__':
    main()
